#include "speechFunctions.h"
#include "debug_macro.h"
#include <iostream>
#include <fstream>
#include <vector>
#include <cstdio>
#include <string>
#include <io.h>
#include <cmath>
#include <set>
#include <map>
#include <utility>
#include <stack>
#include <algorithm>
#include <exception>

#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/convenience.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/format.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/random.hpp>
//#include <boost/thread.hpp>

namespace bfs = boost::filesystem;
using namespace std;
using namespace bfs;


#define TRAIN_RATIO 0.7
#define TEST_RATIO 0.2
#define CV_RATIO 0.1



//////class Options definations//////////////////////////////////////////////////////

Options::Options()
{
	;
}

Options::~Options()
{
	;
}

int Options::registration_options(const string& option_key_)
{
	if (option_key_.empty()){
		std::cout << "warning : unsuitable value for option registration \r\n";
		return -1;
	}
	if (! find_options(option_key_)){
		options_[option_key_] = "";
	}else{
		return 0;
	}
}


int Options::add_options(const string& option_key_, const string& option_value_)
{
	if (option_key_.empty() || option_value_.empty()){
		std::cout << "warning : unsuitable value for option adding \r\n";
		return -1;
	}
	if (find_options(option_key_)){
		options_[option_key_] = option_value_;
	}else{
		std::cout << "warning : wrong option key for options \r\n";
		return -1;
	}
	return 0;
}

bool Options::find_options(const string& option_key_) const
{
	std::map<string, string>::const_iterator option_it = options_.find(option_key_);
	if (option_it != options_.end()){
		return true;
	}
	return false;
}



string Options::get_option_value(const string& option_key_)
{
	string key_ = ""; key_.assign(option_key_.data());
	if (key_.empty()){
		return "";
	}
	if (find_options(key_)){
		string value_; value_.assign(options_[key_]);
		return value_;
	}else
		return "";
}

//////////////////////////////////////////////////////////////////////////


WAVE::WAVE(const char* filename, const char* wave_id)
{
	if (filename && wave_id)
	{
		m_waveHeader = ReadWaveHeader(filename);
		m_waveData = ReadWaveDate(filename);
		string tmp_out_wave = filename;
		tmp_out_wave.append(".wav");
		FILE* tmp_fp = fopen(tmp_out_wave.data(), "wb");
		WriteWaveHeader(m_waveHeader, tmp_fp);
		fwrite(m_waveData, sizeof(short), m_waveHeader->data_size/2, tmp_fp);
		fclose(tmp_fp);

		strcpy(m_waveId, wave_id);
		m_waveId[strlen(m_waveId)+1] = '\0';
		strcpy(m_waveDir, filename);
		m_waveDir[strlen(m_waveDir)+1] = '\0';

	}else{
		m_waveData = 0;
		m_waveHeader = 0;
		m_waveId[0] = '\0';
		m_waveDir[0] = '\0';
	}

}
WAVE::~WAVE()
{
	if (m_waveHeader)
	{
		delete m_waveHeader;m_waveHeader = 0;

	}
	if (m_waveData)
	{
		delete m_waveData;m_waveData =0;
	}
}

WAV_HEADER* WAVE::getWaveHeader()
{
	return m_waveHeader;
}

void WAVE::GetFileName(const char* complete_dir, char* single_filename)
{
	ASSERT(complete_dir && single_filename);
	using namespace boost;
	vector<string> dirUnits;
#ifdef WIN32
	boost::algorithm::split(dirUnits, complete_dir, boost::algorithm::is_any_of("\\"),boost::token_compress_on);
#else
	boost::algorithm::split(dirUnits, complete_dir, boost::algorithm::is_any_of("\/"),boost::token_compress_on);
#endif
	if(! dirUnits[dirUnits.size()-1].empty()){
		strcpy(single_filename, dirUnits[dirUnits.size()-1].data());
	}
}


int WAVE::DoSpeechSplit(unsigned subseg_id, float start, float end, 
	float offset, unsigned sample_rate, const char* output_dir)
{
	int original_seg_end = end;
	start += offset;
	if (start < 0)
	{
		start  = 0;
	}
	end += offset;
	if (end < 0){
		end = original_seg_end;
	}
	char originWaveName[512] = {'\0'};
	GetFileName(GetWaveDir(), originWaveName);
	char subWaveName[512] = {'\0'};
	sprintf(subWaveName, "%s_%08d.wav", originWaveName, subseg_id);
	std::string completeOutputDir = output_dir;
	completeOutputDir.append("\\");
	completeOutputDir.append(originWaveName);
	MakeDirectory(completeOutputDir.data());
	completeOutputDir.append("\\");
	completeOutputDir.append(subWaveName);
	WAV_HEADER subSegHeader = *(getWaveHeader());
	subSegHeader.sample_rate = sample_rate;
	
	FILE* fpOutput = fopen(completeOutputDir.data(),"wb");
	if( NULL == fpOutput)
	{
		printf("Output sub wave can not be open !\n");
		return -1;
	}

	int nSample = getWaveHeader()->data_size/sizeof(short);
	float totalTime = 1.0 * nSample/subSegHeader.sample_rate;
	float durTime = end - start;

	if(start > totalTime)
	{
		printf("%s the start time is larger than the total time\n",GetWaveDir());
		fclose(fpOutput);
		return -1;
	}

	if( (end < 1e-6)  &&  (durTime > 1e-6)  )
	{
		end = start + durTime;
	}
	else if( (durTime < 1e-6)  &&  (end > 1e-6)  )
	{
		durTime = end - start;
	}
	else
	{
		//end = nSample / subSegHeader.sample_rate;
		durTime = end - start;
	}

	if(end > totalTime)
	{
		end = totalTime;
	}
	if(durTime + start > totalTime)
	{
		durTime = end - start;
	}

	short * data_pointer = GetWaveData();
	if(data_pointer == NULL)
	{
		printf("out of memory\n");
		return -1;
	}

	int startIndex = start * getWaveHeader()->sample_rate;
	int endIndex = end * getWaveHeader()->sample_rate;
	int durIndex = durTime * getWaveHeader()->sample_rate;
	subSegHeader.file_size = sizeof(WAV_HEADER) + durIndex * sizeof(unsigned short);
	subSegHeader.data_size = durIndex * sizeof(short);

	WriteWaveHeader(&subSegHeader, fpOutput);
	fwrite(&data_pointer[startIndex],sizeof(unsigned short),durIndex,fpOutput);

	fclose(fpOutput);	fpOutput = NULL;
	return 0;
}

WAV_HEADER* WAVE::ReadWaveHeader(const char* file_name)
{
	if (! file_name)
	{
		return 0;
	}
	if (! IsFileAWave(file_name))
	{
		return 0;
	}
	FILE* fp = fopen(file_name, "rb");
	WAV_HEADER* currHeader = new WAV_HEADER;

	fread(currHeader[0].riff_id, sizeof(char), 4, fp); //     'RIFF'

	if( (currHeader[0].riff_id[0] == 'R')    &&   (currHeader[0].riff_id[1] == 'I')  &&  
		(currHeader[0].riff_id[2] == 'F')    &&  (currHeader[0].riff_id[3] == 'F' )  )
	{ 
		fread(&currHeader[0].file_size, sizeof(unsigned int), 1, fp); //             
		fread(currHeader[0].wave_id, sizeof(char), 4, fp);         //     'WAVE'


		if (  (currHeader[0].wave_id[0] == 'W')    &&   (currHeader[0].wave_id[1] == 'A')  &&  
			(currHeader[0].wave_id[2] == 'V')    &&  (currHeader[0].wave_id[3] == 'E'  ) )
		{ 
			fread(currHeader[0].fmt_id, sizeof(char), 4, fp);     //     4     "fmt ";
			fread(&currHeader[0].format_length, sizeof(unsigned int),1,fp);
			fread(&currHeader[0].format_tag, sizeof(short), 1, fp); //         tag
			fread(&currHeader[0].channels, sizeof(short),1,fp);    //          x  
			fread(&currHeader[0].sample_rate, sizeof(unsigned int), 1, fp);   //               
			fread(&currHeader[0].avg_bytes_sec, sizeof(unsigned int), 1, fp); //          x    
			fread(&currHeader[0].block_align, sizeof(short), 1, fp);     //           
			fread(&currHeader[0].bits_per_sample, sizeof(short), 1, fp);       //                 
			fread(currHeader[0].data_id, sizeof(char), 4, fp);                      //     'data'
			fread(&currHeader[0].data_size, sizeof(unsigned int), 1, fp);     //      x      
		}
		else
		{
			printf("Error: RIFF file but not a wave file\n");
			fclose(fp);
			return 0;
		}
	}
	else
	{
		printf("Error: not a RIFF file\n");
		fclose(fp);
		return 0;
	}
	fclose(fp);
	return currHeader;
}

short* WAVE::ReadWaveDate(const char* file_name)
{
	ASSERT(file_name)
	bool isWave = false;
	if (IsFileAWave(file_name))
	{
		isWave = true;
	}
	FILE* fp = fopen(file_name, "rb");
	long long dataSize = 0;
	fseek(fp, 0, SEEK_END);
	if (isWave)
	{
		dataSize = ftell(fp) - 44;
	}else{
		dataSize = ftell(fp);
	}
	fseek(fp, 44, SEEK_SET);
	short* data = new short[dataSize/2];
	fread(data, sizeof(short), dataSize/2, fp);
	fclose(fp);
	return data;
}

int WAVE::WriteWaveHeader(const WAV_HEADER* wave_header, FILE* fp)
{
	if(fp)
	{
		//fwrite(&wave_header,sizeof(WAV_HEADER),1,fp);
		fwrite(&(wave_header->riff_id), sizeof(char), 4, fp); 
		fwrite(&(wave_header->file_size), sizeof(unsigned int), 1, fp);
		fwrite(wave_header->wave_id, sizeof(char), 4, fp);

		fwrite(wave_header->fmt_id, sizeof(char), 4, fp);     // 4     "fmt ";
		fwrite(&(wave_header->format_length), sizeof(unsigned int),1,fp);
		fwrite(&(wave_header->format_tag), sizeof(short), 1, fp); //     tag
		fwrite(&(wave_header->channels), sizeof(short),1,fp);    //     x  
		fwrite(&wave_header->sample_rate, sizeof(unsigned int), 1, fp);   //          
		fwrite(&wave_header->avg_bytes_sec, sizeof(unsigned int), 1, fp); //      x    
		fwrite(&wave_header->block_align, sizeof(short), 1, fp);     //       
		fwrite(&wave_header->bits_per_sample, sizeof(short), 1, fp);       //             
		fwrite(&wave_header->data_id, sizeof(char), 4, fp);                  //     'data'
		fwrite(&wave_header->data_size, sizeof(unsigned int), 1, fp);     //  x      

		return 0;
	}
	return -1;
}

SpeechAnalysis::SpeechAnalysis():m_wave(0)
{
	;
}

SpeechAnalysis::~SpeechAnalysis()
{
	if (m_wave)
	{
		delete m_wave;m_wave = 0;
	}
	;
}


/////////////////////////// Global functions /////////////////////////////////////////////

int PrintVectorInfo(const std::vector<int>& in_vector_)
{
	int ret = -1;
	if (in_vector_.size() == 0){
		return ret;
	}
	for (std::vector<int>::const_iterator int_it = in_vector_.begin(); int_it != in_vector_.end();++int_it){
		std::cout << *int_it << std::endl;
	}
	ret = 0;
	return ret;
}


bool IsFileAWave(const char* file_name)
{
	if(! file_name){
		std::cout << "speech file name is empty !" << std::endl;
		exit(-1);
	}
	FILE* fp = fopen(file_name, "rb");
	if(! fp){
		std::cout << "speech file can not be open !" << std::endl;
		return false;
	}
	char fileHeader[5] = {'\0'};
	fread(fileHeader,sizeof(char), 4, fp);
	fclose(fp);
	if (strcmp(fileHeader, "RIFF") == 0)
	{
		return true;
	}
	return false;

}

bool MakeDirectory(const std::string& dir_name)
{
	using namespace bfs;
	if(dir_name.empty())return false;
	std::string strPath = dir_name;
	bfs::path dirPath(strPath);

	if (! bfs::is_directory(dirPath))
	{
		if (bfs::is_regular_file(dirPath))
		{
			strPath.append("_seg");
			bfs::path newPath(strPath);//path is initialized by string 
			return bfs::create_directories(newPath);
		}
		return bfs::create_directories(dirPath);
	}
	return false;
}


std::vector<char> ParseString2Chars(const string& str_in_)
{
	std::vector<char> charArray;
	string charPtr = "";
	charPtr.assign(str_in_.data());
	if (charPtr.size() == 0)
	{
		return charArray;
	}
	
	for (int i = 0; i < str_in_.size();++i)
	{
		if (charPtr.at(i) != '\0')
		{
			charArray.push_back(charPtr[i]);
		}
	}
	return charArray;
}


std::vector<std::string> SplitString(const std::string& str_in_, const string& str_sep_, int split_num)
{
	std::vector<string> split_results;
	split_results.resize(0);
	if (str_in_.empty())
	{
		return split_results;
	}
	std::string str;
	str.assign(str_in_.data());
	std::vector<char> charSeps;
	if (! str_sep_.empty())
	{
		charSeps = ParseString2Chars(str_sep_);
		ASSERT(charSeps.size() > 0);
	}
	std::set<char> strSeperators(charSeps.begin(), charSeps.end());
	int part_index = 1; //at least one part for this string
	boost::algorithm::trim_if(str, boost::algorithm::is_any_of(" \r\n") );

	while(true){
		std::string::size_type blank_pos =str.find_first_of(" \t");
		if(blank_pos != std::string::npos)
		{
			std::string first_part = str.substr(0, 0+blank_pos);
			split_results.push_back(first_part);
			while(str.at(blank_pos) == ' ' || str.at(blank_pos) == '\t') blank_pos++;
			str.assign(str.substr(blank_pos));
			part_index++;
		}else
		{
			break;
		}
		if (split_num > 0 && part_index >= split_num)
		{
			break;
		}
		
	}
	split_results.push_back(str);
	return split_results;
}



WAV_HEADER* MakeWaveHeader(const unsigned long file_size_, unsigned short channel_nums_,
	unsigned long sample_rate_, unsigned long avg_bytes_sec, unsigned short bits_per_sample_)
{
	//size of block is : 44 chars
	WAV_HEADER* wave_header = new WAV_HEADER;
	memset(wave_header, 0, sizeof(WAV_HEADER));
	wave_header[0].riff_id[0] = 'R';
	wave_header[0].riff_id[1] = 'I';
	wave_header[0].riff_id[2] = 'F';
	wave_header[0].riff_id[3] = 'F';
	// 4 chars for storing 'RIFF'.4 chars and one '\0' can be useful for using strcmp
	wave_header[0].file_size = file_size_ + 44 -8;	// int type, size of file
	//"WAVE" notification
	wave_header[0].wave_id[0] = 'W';
	wave_header[0].wave_id[1] = 'A';
	wave_header[0].wave_id[2] = 'V';
	wave_header[0].wave_id[3] = 'E';
	//"fmt ", 4 chars
	wave_header[0].fmt_id[0] = 'f';
	wave_header[0].fmt_id[1] = 'm';
	wave_header[0].fmt_id[2] = 't';
	wave_header[0].fmt_id[3] = ' ';

	wave_header[0].format_length = 16; // 16 bits data
	wave_header[0].format_tag = 1;//short, for pcm, it is equal to 1

	wave_header[0].channels = channel_nums_;//channel number
	wave_header[0].sample_rate = sample_rate_; //sample rate
	wave_header[0].avg_bytes_sec = avg_bytes_sec;
	wave_header[0].block_align = channel_nums_ * 16/8;
	wave_header[0].bits_per_sample = bits_per_sample_;


	wave_header[0].data_id[0] = 'd';
	wave_header[0].data_id[1] = 'a';
	wave_header[0].data_id[2] = 't';
	wave_header[0].data_id[3] = 'a';
	wave_header[0].data_size = file_size_ ;

	return wave_header;
}


int GenerateIntRandom(std::vector<int>& results_, const int results_size_)
{
	using namespace boost::random;
	if (results_size_ <= 0){
		return -1;
	}
	boost::random::mt19937 rng(time(0));  
	std::map<int, int> random_pool;
#ifdef LOGTXT
	std::ofstream log("log.txt", ios::out);
#endif
	//uniform_int
	boost::uniform_int<> real(0, results_size_-1);
	while (results_.size()<results_size_)  
	{
		int new_random_int = real(rng);
		std::map<int,int>::iterator num_it = random_pool.find(new_random_int);
		if (num_it == random_pool.end()){
			results_.push_back( new_random_int );
			random_pool.insert(std::make_pair(new_random_int, 1));
#ifdef LOGTXT
			log << new_random_int << std::endl;
#endif
		} 

	}
	fprintf(stdout, "the size of random number : %d\r\n", results_.size());
#ifdef LOGTXT
	log.close();
#endif
	return 0;
}


int computeIntersection(const std::map<string,string>& aContainer, const std::map<string, string>& bContainer, std::vector<string>& resultContainer)
{
	int intersectionSize = -1;
	if (aContainer.size()<=0 || bContainer.size()<=0 ){
		return intersectionSize;
	}
	resultContainer.erase(resultContainer.begin(), resultContainer.end());
	fprintf(stdout, "the size of first set : %d\r\n", aContainer.size());
	fprintf(stdout, "the size of second set : %d\r\n", bContainer.size());
	int compareTimes  = 0;
	for (std::map<string,string>::const_iterator str_it = aContainer.begin();str_it != aContainer.end();++str_it){
		std::map<string, string>::const_iterator b_it = bContainer.find( str_it->first);

		if ( b_it != bContainer.end() ){
			resultContainer.push_back(str_it->first);
		}else{
			std::cout << "not found\r\n";
		}
	}
	intersectionSize = resultContainer.size();
	return intersectionSize;
}

int SplitKaldiScp(Options& paragram_options_)
{
	//This is a multiThread function, and boost thread is used here.
	string kaldi_wave_scp_ = paragram_options_.get_option_value("--scp-in");
	string kaldi_text_scp_ = paragram_options_.get_option_value("--text-in");
	if (kaldi_wave_scp_.empty() || kaldi_text_scp_.empty()) return -1;
	float train_set_ratio = TRAIN_RATIO, test_set_ratio = TEST_RATIO, cv_set_ratio = CV_RATIO;
	if (  paragram_options_.get_option_value("--train-ratio").compare("") > 0){
		train_set_ratio = boost::lexical_cast<float>(paragram_options_.get_option_value("--train-ratio"));
	}
	if (  paragram_options_.get_option_value("--test-ratio").compare("") > 0){
		test_set_ratio =  boost::lexical_cast<float>(paragram_options_.get_option_value("--test-ratio"));
	}
	if (  paragram_options_.get_option_value("--cv-ratio").compare("") > 0){
		cv_set_ratio = boost::lexical_cast<float>(paragram_options_.get_option_value("--cv-ratio"));
	}

	try
	{
		assert( (train_set_ratio + test_set_ratio + cv_set_ratio) <=1.0 );
	}
	catch (std::exception& e)
	{
		std::cerr << "exception caught: " << e.what() << "\r\n";
	}
	
	std::ifstream wave_fp(kaldi_wave_scp_.data(), ios::in);
	if(! wave_fp.good()) return -1;
	////////start to process the wave scp here //////////////////////////////////////////////
	std::map<string, string> wave_scp_, text_scp_;

	string scpline;
	std::vector<string> scp_line_units_;
	while(std::getline(wave_fp, scpline, '\n'))
	{
		if (! scpline.empty())
		{
			//open the no header file and adder header
			boost::algorithm::split(scp_line_units_, scpline, boost::algorithm::is_any_of(" \t"), boost::algorithm::token_compress_on);
			const string wave_file_id_ = scp_line_units_[0];
			const string wave_file_name_ = scp_line_units_[scp_line_units_.size()-1];
			if (wave_scp_.empty())
			{
				string wavID =  boost::to_lower_copy(wave_file_id_) ;
				wave_scp_.insert( std::make_pair(wavID, wave_file_name_) );
			}else{
				string wavID =  boost::to_lower_copy(wave_file_id_) ;
				std::map<string,string>::const_iterator set_it = wave_scp_.find(wavID);
				if (set_it == wave_scp_.end() )
				{			
					wave_scp_.insert( std::make_pair(wavID, wave_file_name_) );
				}
			}
		}
	}
	wave_fp.close();
	std::cout << "The number of utterance : " << wave_scp_.size() << std::endl;
	////////start to process the wav scp here //////////////////////////////////////////////
	std::ifstream text_fp(kaldi_text_scp_.data(),ios::in);
	if (text_fp.good())
	{
		while(std::getline(text_fp, scpline, '\n'))
		{
			if (! scpline.empty())
			{
				//open the no header file and adder header
				boost::algorithm::split(scp_line_units_, scpline, boost::algorithm::is_any_of(" \t"), boost::algorithm::token_compress_on);
				const string text_file_id_ = scp_line_units_[0];
				std::reverse(scp_line_units_.begin(), scp_line_units_.end());
				scp_line_units_.pop_back();
				std::reverse(scp_line_units_.begin(), scp_line_units_.end());
				const string text_content_ = boost::join(scp_line_units_, " ");
				if (text_scp_.empty())
				{
					string textID = boost::to_lower_copy(text_file_id_);
					text_scp_.insert( std::make_pair(textID, text_content_) );
				}else{
					string textID = boost::to_lower_copy(text_file_id_);
					std::map<string,string>::const_iterator set_it = text_scp_.find(textID);
					if (set_it == text_scp_.end() )
					{
						text_scp_.insert( std::make_pair(textID, text_content_) );
					}
				}
			}
		}
	}
	text_fp.close();
	std::cout << "The number of trans : " << text_scp_.size() << std::endl;
	std::vector<string> uniqueNames;
	int intersize = computeIntersection(wave_scp_, text_scp_, uniqueNames);
	boost::format sizeInfo("The intersection size between scp data and text data is : |%d|\r\n");
	sizeInfo % intersize;
	std::cout << sizeInfo;
	//////generate random number with to split data into training set, test set and cv set////////////////////////////////
	std::vector<int> randomNumIndice;
	GenerateIntRandom(randomNumIndice, intersize); //0-->(size-1)

	//start to split all, train set : 70%, test set : 20%, cv set : 10 % //////////////////////////////////////////
	string split_train_set_ = "split_train_set.scp", split_train_text = "split_train_set.text";
	string split_test_set_ = "split_test_set.scp", split_test_text = "split_test_set.text";
	string split_cv_set_ = "split_cv_set.scp", split_cv_text = "split_cv_set.text";
	int trainSetSize = intersize * train_set_ratio, testSetSize = intersize * test_set_ratio;
	int cvSetSize = intersize - trainSetSize - testSetSize;
	try{
		ASSERT(cvSetSize >= 0);
	}catch(std::exception& e){
		std::cout << "cv set size : " << e.what() << std::endl;
	}
	
	fprintf(stdout, "The results : train set: %d, test set: %d, cv set: %d. \r\n", trainSetSize, testSetSize, cvSetSize);
	//////////////////////////////////////////////////////////////////////////
	ofstream train_fp(split_train_set_.data(), ios::out);
	ofstream test_fp(split_test_set_.data(), ios::out);
	ofstream cv_fp(split_cv_set_.data(), ios::out);
	//////////////////////////////////////////////////////////////////////////
	ofstream train_text_fp(split_train_text.data(), ios::out);
	ofstream test_text_fp(split_test_text.data(), ios::out);
	ofstream cv_text_fp(split_cv_text.data(), ios::out);
	//////////////////////////////////////////////////////////////////////////
	
	if (! train_fp.good() || ! test_fp.good() || ! cv_fp.good() || ! train_text_fp.good() || ! test_text_fp.good() || ! cv_text_fp.good()){
		ERROR("Out file open error !");
		return -1;
	}
	std::vector<string> trainIDs, testIDs, cvIDs;
	for (int i=0;i<randomNumIndice.size();++i){
		int ind = randomNumIndice[i];
		string scpId = uniqueNames[ind];
		if (i<trainSetSize){
			trainIDs.push_back(scpId);
		}else if (i >= trainSetSize && i < (trainSetSize+testSetSize)){
			testIDs.push_back(scpId);
		}else{
			cvIDs.push_back(scpId);
		}
	}
	/////////////////////sort all scp ids ///////////////////////////////////////////
	std::sort(trainIDs.begin(), trainIDs.end());
	std::sort(testIDs.begin(), testIDs.end());
	std::sort(cvIDs.begin(), cvIDs.end());
	
	//output data into 6 different files
	for (int i=0;i<trainIDs.size();++i){
		string currFileId = trainIDs[i];
		train_fp << currFileId << "    " << wave_scp_[currFileId] << std::endl;
		train_text_fp << currFileId << "    " << text_scp_[currFileId] << std::endl;
	}
	for (int i=0;i<testIDs.size();++i){
		string currFileId = testIDs[i];
		test_fp << currFileId << "    " << wave_scp_[currFileId] << std::endl;
		test_text_fp << currFileId << "    " << text_scp_[currFileId] << std::endl;
	}
	for (int i=0;i<cvIDs.size();++i){
		string currFileId = cvIDs[i];
		cv_fp << currFileId << "    " << wave_scp_[currFileId] << std::endl;
		cv_text_fp << currFileId << "    " << text_scp_[currFileId] << std::endl;
	}
	//////////close all files //////////////////////////////////////////////////////
	train_fp.close();
	test_fp.close();
	cv_fp.close();
	train_text_fp.close();
	test_text_fp.close();
	cv_text_fp.close();
	////////end of processing //////////////////////////////////////////////
	return 0;
}


//thread funciton for adding header to raw pcm speech file
SpeechProcessState::ERROR_TYPE AddHeader4Utterance(const string& utterance_file_)
{
	//SpeechProcessState::ERROR_TYPE a = SpeechProcessState::ERROR_TYPE.CORRECT_STATE;

	if(utterance_file_.empty() ) return SpeechProcessState::ERROR_OTHERS;
	if ( ! IsFileAWave(utterance_file_.data()))
	{
		fprintf(stdout, "Adding wave header for %s\r\n", utterance_file_.data());
		FILE* wave_fp = fopen(utterance_file_.data(), "rb");
		if (wave_fp == NULL){
			fclose(wave_fp);return SpeechProcessState::ERROR_READ_FILE;
		}
		fseek(wave_fp, 0, SEEK_END);
		long int data_size = ftell(wave_fp);
		fseek(wave_fp, 0, SEEK_SET);
		short* data_buf = new short[data_size/2];
		ASSERT(data_buf != NULL);
		memset(data_buf, 0, data_size);
		if( ! data_size/2 == fread(data_buf, sizeof(short),data_size/2, wave_fp) )
		{
			std::cout << "Read wave error !\r\n";
			return SpeechProcessState::ERROR_WRITEN_FILE;
		}
		WAV_HEADER* a_header = MakeWaveHeader(data_size, 1, 8000, 16000, 16);
		fclose(wave_fp);

		string headered_wave = utterance_file_ + ".wav";
		FILE* headered_fp = fopen(headered_wave.data(), "wb");
		if (headered_fp == NULL){
			fclose(headered_fp);return SpeechProcessState::ERROR_WRITEN_FILE;
		}

		WAVE void_wave(NULL, NULL);
		void_wave.WriteWaveHeader(a_header, headered_fp);
		fwrite(data_buf, sizeof(short), data_size/2, headered_fp);
		fclose(headered_fp);

		delete data_buf;
		data_buf = 0;
		if (a_header){
			delete a_header;a_header = 0;
		}
	}
	return SpeechProcessState::CORRECT_STATE;

}


SpeechProcessState::ERROR_TYPE enumGlobalVar(const string& )
{
	SpeechProcessState::ERROR_TYPE rst = SpeechProcessState::CORRECT_STATE;
	return rst;
}


int ProcessTimitTextTrans(Options& paragram_options_)
{
	string trans_flist_ = paragram_options_.get_option_value("--text-in");
	int num_entries=0, num_succ = 0;
	if (access(trans_flist_.data(), 04) < 0){
		return num_succ;
	}
	try
	{
		std::ifstream list_ifs(trans_flist_.data(), ios::in);
		if (list_ifs.is_open())
		{
			std::string transLine;
			std::vector<string> trLineUnits;
			while (getline(list_ifs, transLine))
			{
				num_entries++;
				if ( ! transLine.empty())
				{
					boost::trim_if(transLine, boost::is_any_of("\r\n"));
					AdjustAnnotation(transLine, 16000, 10, 25);
					if ( ! ProcAnnoatation(transLine))
					{
						std::cerr << "annotation transformation is not successful \r\n";
						exit(-1);
					}
					num_succ++;
					if (num_succ % 10 == 0){
						std::cout << "Process : " << num_succ << " utterances .\r\n";
					}
					
				}
				
			}
			
		}
		
		list_ifs.close();
	}
	catch (exception* e)
	{
		std::cout << e->what() << std::endl;
	}
	return num_succ;
}


bool ProcAnnoatation(const string& a_annotation_file_)
{
	int tranform_succ = 0, num_lines=0;
	ASSERT(!a_annotation_file_.empty());
	std::ifstream annotation_ifs(a_annotation_file_.data(), ios::in);
	std::vector<std::vector<string> > feats_labels;
	try{
		string annotationLine; std::vector<string> annotationUnits;
		while (getline(annotation_ifs, annotationLine))
		{
			if (! annotationLine.empty())
			{
				num_lines++;
				annotationUnits = SplitString(annotationLine, "\t ", 3);
				if (annotationUnits.size() == 3)
				{
					float segStart = 0, segEnd = 0;
					string segAdHoc = annotationUnits[2];
					segStart = boost::lexical_cast<float>(annotationUnits[0]);
					segEnd = boost::lexical_cast<float>(annotationUnits[1]);
					boost::algorithm::trim_if(segAdHoc, boost::algorithm::is_any_of("[]"));
					std::vector<string> labelarray;
					labelarray = SplitString(segAdHoc, " ", -1);
					annotationUnits.pop_back();
					for (int i = 0; i < labelarray.size();++i)
					{
						annotationUnits.push_back(labelarray[i]);
					}
					feats_labels.push_back(annotationUnits);
					
					tranform_succ++;
				}
				
			}
			
		}
	}
	catch (exception& e){
		std::cout << e.what() << std::endl;
		annotation_ifs.close();
	}
	annotation_ifs.close();
	string::size_type sep_pos = a_annotation_file_.find_last_of(".");
	string label_file_name = "";
	if (sep_pos != string::npos)
	{
		label_file_name.assign(a_annotation_file_.substr(0, sep_pos));
		label_file_name.append(".label");
	}
	
	WriteVectorWithHtkMode(label_file_name, feats_labels);
	
	
	return tranform_succ == num_lines;
}


int WriteVectorWithHtkMode(const string& out_file_name_, const std::vector<std::vector<string> >& to_write_)
{
	int write_succ = 0;
	ASSERT(!out_file_name_.empty() && to_write_.size() > 0);

	HTKHeader this_header;
	const std::vector<string>& lastVec = to_write_[to_write_.size() - 1];
	this_header.sample_period = 100;
	this_header.singleVectorCharsNum = lastVec.size() * sizeof(int);
	this_header.sampleKind = 9; //User defined 
	float lastSamplePoint = boost::lexical_cast<float>(lastVec[1].data());
	int total_frame_num = 0;
	std::vector<int> labels;
	try
	{
		std::vector<std::vector<string> >::const_iterator i_it = to_write_.begin(), end = to_write_.end();
		
		for (; i_it != end;++i_it)
		{
			float seg_start = boost::lexical_cast<float>(i_it->at(0));
			float seg_end = boost::lexical_cast<float>(i_it->at(1));
			int seg_frame_num = 0;// = ComputeFrameNum(seg_end - seg_start, 16000, 10, 25);
			//total_frame_num += seg_frame_num;
			for (int i = 0; i < seg_frame_num;++i)
			{
				for (int j = 2; j < i_it->size();j++)
				{
					labels.push_back(boost::lexical_cast<int>( i_it->at(j) ) );
				}
			}
		}
	}
	catch (exception& e)
	{
		std::cout << e.what() << std::endl;
	}
	this_header.feature_total_num = total_frame_num;
	FILE* out_fp = fopen(out_file_name_.data(), "wb");
	ASSERT(out_fp != NULL);
	fwrite(&(this_header), 1, sizeof(HTKHeader), out_fp);
	for (int i = 0; i < labels.size();++i)
	{
		int this_label = labels[i];
		fwrite(&(this_label), 1, sizeof(int), out_fp);
	}
	fclose(out_fp);
	return write_succ;
}

int ComputeFrameNum(float seg_start_, float seg_end_, float sample_rate_, float frame_shift_, float frame_dur_)
{
	ASSERT(seg_end_ >= seg_start_);
	int frame_sample_num = sample_rate_ * frame_shift_ / 1000;
	int frame_num = 0;
	if ((seg_end_-seg_start_)<=frame_sample_num)
	{
		std::cout << "too small duration of segment !\r\n";
		return frame_num;
	}
	
	
	
	return frame_num;
}


bool AdjustAnnotation(const string& annotation_file_name_, float sample_rate_, float frame_shift_, float frame_dur_)
{
	ASSERT(!annotation_file_name_.empty());
	if (sample_rate_ <= 0 || frame_shift_ <= 0 || frame_dur_<=0 || frame_shift_ >= frame_dur_)
	{
		std::cout << "parameter setting error !\r\n";
		return false;
	}
	std::ifstream an_os(annotation_file_name_.c_str(), ios::in);
	string annotation_line = "";
	std::vector<string> lineUnits;
	std::vector<float> seg_times;
	std::vector<string> seg_labels;
	while (std::getline(an_os, annotation_line, '\n'))
	{
		boost::trim_if(annotation_line, boost::is_any_of("\r "));
		if (! annotation_line.empty())
		{
			lineUnits = SplitString(annotation_line, " \t", 3);
			if (lineUnits.size() > 2)
			{
				seg_times.push_back(boost::lexical_cast<float>(lineUnits[0]) );
				seg_times.push_back(boost::lexical_cast<float>(lineUnits[1]) );
				seg_labels.push_back(lineUnits[2]);
			}
		}
	}
	an_os.close();
	FindBestBoundary(seg_times);
	// using frame length and frame duration to reassign borders of these segments [8/18/2015 Administrator]
	float seg_offset = 0.0f;
	bool is_padding_start = false;
	bool is_padding_end = false;
	for (int i = 0; i < seg_times.size()-1;i+=2)
	{
		int samples_of_frame = frame_dur_ * sample_rate_ / 1000;
		int samples_of_shift = frame_shift_ * sample_rate_ / 1000;
		if (i == 0)
		{
			int this_dur = seg_times[i + 1] - seg_times[i];
			std::pair<int, int> rst = ComputeMinOffSet(this_dur, samples_of_shift, samples_of_frame);
			//first segment of annotation
			seg_times[i + 1] = seg_times[i+1] + rst.second;
			continue;
		}
		if (i > 0) //annotation where allocated in the middle of utterance
		{
			if (seg_times[i] != seg_times[i-1])
			{
				seg_times[i] = seg_times[i - 1];
			}
			int this_dur = seg_times[i + 1] - seg_times[i];
			std::pair<int, int> rst = ComputeMinOffSet(this_dur, samples_of_shift, samples_of_frame);
			seg_times[i + 1] = seg_times[i + 1] + rst.second;
			seg_offset = rst.second;
		}
	}
	//start to output to disk using the reassigned time
	string new_annotation;
	std::string::size_type last_point = annotation_file_name_.find_last_of(".");
	if (last_point != string::npos)
	{
		new_annotation.assign(annotation_file_name_.substr(0, last_point));
		new_annotation.append(".stdphn");
	}
	std::ofstream phn_os(new_annotation.c_str(), ios::out);
	if (phn_os.is_open())
	{
		for (int i = 0; i < seg_labels.size();++i)
		{
			phn_os << seg_times[i] << " " << seg_times[i + 1] << " " << seg_labels[i] << std::endl;
		}
	}
	
	phn_os.close();
	
	return true;
}


std::pair<int, int> ComputeMinOffSet(int total_num, int unit_num, int max_renmain)
{
	std::pair<int, int> rst = std::make_pair(0, 0);
	if (total_num<max_renmain)
	{
		return rst;
	}
	int max_divide = 1;
	int speech_length = ComputeSpeechLengthByFrame(unit_num, max_divide, max_renmain);
	while (speech_length < total_num)
	{
		max_divide++;
		speech_length = ComputeSpeechLengthByFrame(unit_num, max_divide, max_renmain);
		
	}
	int greater_length = speech_length - total_num;
	int less_length = total_num - (speech_length - unit_num);
	if (greater_length < less_length) // padding samples frame next segment [8/18/2015 Administrator]
	{
		rst.second = greater_length;
		rst.first = max_divide;
	} else{ // delete samples in the end of this segment [8/18/2015 Administrator]
		rst.first = max_divide - 1;
		rst.second = -less_length;

	}
	return rst;
}


int FindBestBoundary(const std::vector<float>& seg_boundaries)
{
	int frame_length = 400, frame_shift = 160;
	bool is_use_layer_pruning = false, is_beam_width_pruning = true;
	if (seg_boundaries.empty())
	{
		return -1;
	}
	std::vector<float> boundaries(seg_boundaries.begin(), seg_boundaries.end());
	std::vector<std::vector<BoundaryInfo> > searchTree;
	searchTree.resize(boundaries.size()/2);
	InitSearchTreeNet(&searchTree, boundaries);
	int node_index = 2; //after initialization, we start node index with 2
	for (int i = 1; i < searchTree.size();i++){//start to search on the first layer of search tree

		searchTree[i].resize(searchTree[i - 1].size() * 2); //resize the size of the layer

		for (int j = 0; j < searchTree[i - 1].size();j++){ //pass through all nodes on previous layer to yield new nodes storing in this layer
			// one node can yield two children nodes here
			const BoundaryInfo& curr_father_node = searchTree[i - 1][j];
			BoundaryInfo* great_child = curr_father_node.Copy();// searchTree[i - 1][j].Copy();
			int seg_length = boundaries[2 * i + 1] - great_child->get_revised_boundary();
			std::pair<float, float> forwardback_objs = ComputeForwardBackObjs(seg_length, frame_shift, frame_length);

			great_child->adding_obj(std::abs(forwardback_objs.first));
			great_child->set_adding_state(true);
			great_child->set_revised_boundary(boundaries[2 * i + 1] + forwardback_objs.first);
			great_child->set_original_boundary(boundaries[2 * i + 1]);
			great_child->set_curr_obj(std::abs(forwardback_objs.first));
			great_child->set_father_node_index(great_child->get_node_index());
			great_child->set_node_index(node_index++);
			searchTree[i][2 * j] = *great_child;
			delete great_child; great_child = NULL;

			BoundaryInfo* less_child = curr_father_node.Copy();
			less_child->adding_obj(std::abs(forwardback_objs.second));
			less_child->set_curr_obj(std::abs(forwardback_objs.second));
			less_child->set_adding_state(false);
			less_child->set_revised_boundary(boundaries[2 * i + 1] + forwardback_objs.second);
			less_child->set_original_boundary(boundaries[2 * i + 1]);
			less_child->set_father_node_index(less_child->get_node_index());
			less_child->set_node_index(node_index++);

			searchTree[i][2 * j + 1] = *less_child;
			delete less_child; less_child = NULL;
		}
		//we need to prune here to maintain the beam is under a relative reasonable quantity
		//tow methods here: 1 the size of last layer can not surpass some number, 2 the number of nodes of global network can not surpass some number
		// the first method
		std::vector<BoundaryInfo>& lastLayer = searchTree[i];
		if (is_use_layer_pruning)
		{
			int nodesLast = 0;
			
			nodesLast = TreeLayerAveragingPrune(lastLayer);
			if (nodesLast == 0)
			{
				std::cout << "no reasonable node is maintained in the last layer !\r\n";
				exit(-1);
			}
		}
		if (is_beam_width_pruning)
		{
			int beam_width = 4;
			beam_width = TreeLayerNodeNumPrune(lastLayer, beam_width);
		}
	}
	std::vector<BoundaryInfo>& lastLayerInfo = searchTree[searchTree.size() - 1];
	std::vector<BoundaryInfo>::iterator last_it = lastLayerInfo.begin(), last_end = lastLayerInfo.end();
	float min_acc_obj = 1e10;
	int best_node_index = 0, best_father_node = 0;
	for (; last_it != last_end;last_it++)
	{
		if (min_acc_obj>last_it->get_obj())
		{
			min_acc_obj = last_it->get_obj();
			best_node_index = last_it->get_node_index();
			best_father_node = last_it->get_father_node_index();
		}
	}
	std::vector<std::vector<BoundaryInfo> >::reverse_iterator tree_it = searchTree.rbegin(), begin = searchTree.rend();
	std::vector<BoundaryInfo> bestTrace;
	for (; tree_it != begin;tree_it++)
	{
		std::vector<BoundaryInfo>::iterator layer_it = tree_it->begin(), layer_end = tree_it->end();
		if (tree_it == searchTree.rbegin()){
			if (layer_it->get_node_index() == best_node_index)
			{
				bestTrace.push_back(*layer_it);
				best_father_node = layer_it->get_father_node_index();
				continue;
			}
		}
		
		for (; layer_it != layer_end;layer_it++)
		{
			if (layer_it->get_node_index() == best_father_node)
			{
				bestTrace.push_back(*layer_it);
				best_father_node = layer_it->get_father_node_index();
			}
		}
	}
	std::reverse(bestTrace.begin(), bestTrace.end());
	std::vector<BoundaryInfo>::iterator tr_it = bestTrace.begin(), tr_end = bestTrace.end();
	for (; tr_it != tr_end; tr_it++)
	{
		std::cout << "old time : " << tr_it->get_prev_boundary() << " new time : " << tr_it->get_revised_boundary() << std::endl;
	}
	return 0;
}


int InitSearchTreeNet(std::vector<std::vector<BoundaryInfo> >* searchTree, const std::vector<float>& seg_boundaries)
{
	int init_state_num = searchTree->at(0).size();
	if (seg_boundaries.empty() || searchTree->size() == 0)
	{
		return init_state_num;
	}
	
	std::vector<float> boundaries(seg_boundaries.begin(), seg_boundaries.end());
	int frame_length = 400, frame_shift = 160;
	int curr_seg_length = boundaries[1] - boundaries[0];
	int max_divide = 1;
	int speech_length = ComputeSpeechLengthByFrame(frame_shift, max_divide, frame_length);
	while (speech_length < curr_seg_length)
	{
		max_divide++;
		speech_length = ComputeSpeechLengthByFrame(frame_shift, max_divide, frame_length);
	}
	float greater_length = std::abs(speech_length - curr_seg_length);
	float less_length = std::abs(curr_seg_length - (speech_length - frame_shift));
	BoundaryInfo greaterBorder;
	greaterBorder.adding_obj(greater_length); 
	greaterBorder.set_adding_state(true);
	greaterBorder.set_curr_obj(greater_length); 
	greaterBorder.set_original_boundary(boundaries[1]);
	greaterBorder.set_revised_boundary(boundaries[1] + greater_length);
	greaterBorder.set_father_node_index(-1);
	greaterBorder.set_node_index(0);

	BoundaryInfo lessBorder;
	lessBorder.adding_obj(less_length); 
	lessBorder.set_adding_state(false);
	lessBorder.set_curr_obj(less_length); 
	lessBorder.set_original_boundary(boundaries[1]);
	lessBorder.set_revised_boundary(boundaries[1] - less_length);
	lessBorder.set_node_index(1);
	lessBorder.set_father_node_index(-1);


	searchTree->at(0).push_back(greaterBorder);
	searchTree->at(0).push_back(lessBorder);

	
	return searchTree->at(0).size();
}


int TreeLayerAveragingPrune(std::vector<BoundaryInfo>& searchTreeLayer)
{
	int remain_size = searchTreeLayer.size();
	if (remain_size == 0)
	{
		return remain_size;
	}
	float acc_objs = 0.0f, averaging_obj = 0.0;
	std::vector<BoundaryInfo>::iterator it = searchTreeLayer.begin(), end = searchTreeLayer.end();
	for (; it != end;it++)
	{
		acc_objs += it->get_obj();
	}
	averaging_obj = acc_objs / remain_size;
	std::vector<BoundaryInfo> shrinkedInfo(searchTreeLayer.begin(), searchTreeLayer.end());
	searchTreeLayer.erase(searchTreeLayer.begin(), searchTreeLayer.end());
	for (std::vector<BoundaryInfo>::iterator  s_it= shrinkedInfo.begin(); s_it != shrinkedInfo.end();)
	{
		if (s_it->get_obj() < averaging_obj)
		{
			searchTreeLayer.push_back(*s_it);
		}
		s_it++;
	}
	return searchTreeLayer.size();
}


int TreeLayerNodeNumPrune(std::vector<BoundaryInfo>& searchTreeLayer, int prune_width)
{
	int remain_size = searchTreeLayer.size();
	if (remain_size == 0 || searchTreeLayer.size()<=prune_width)
	{
		return remain_size;
	}
	float acc_objs = 0.0f, averaging_obj = 0.0;
	std::vector<BoundaryInfo> shrinkedInfo(searchTreeLayer.begin(), searchTreeLayer.end());
	searchTreeLayer.erase(searchTreeLayer.begin(), searchTreeLayer.end());
	std::vector<BoundaryInfo>::iterator s_it = shrinkedInfo.begin(), end = shrinkedInfo.end();
	std::vector<float> objs;
	objs.resize(shrinkedInfo.size());
	for (int i = 0; s_it != end; s_it++)
	{
		objs[i] = s_it->get_obj();
		i++;
	}
	std::sort(objs.begin(), objs.end());
	float prune_threshold = objs[prune_width - 1];
	std::vector<BoundaryInfo> info;
	for (s_it = shrinkedInfo.begin(); s_it != end;s_it++)
	{
		if (s_it->get_obj()<=prune_threshold)
		{
			info.push_back(*s_it);
		}
	}
	searchTreeLayer = info;
	return searchTreeLayer.size();
}


std::pair<float, float> ComputeForwardBackObjs(int total_num, int unit_num, int max_renmain)
{
	std::pair<float, float> rst = std::make_pair(0.0, 0.0);
	if (total_num < max_renmain)
	{
		return rst;
	}
	int max_divide = 1;
	int speech_length = ComputeSpeechLengthByFrame(unit_num, max_divide, max_renmain);
	while (speech_length < total_num)
	{
		max_divide++;
		speech_length = ComputeSpeechLengthByFrame(unit_num, max_divide, max_renmain);
	}
	int greater_length = speech_length - total_num;
	int less_length = total_num - (speech_length - unit_num);
	rst.first = greater_length;
	rst.second = -less_length;
	return rst;
}